package org.easy.tool.common;

import com.huaban.analysis.jieba.JiebaSegmenter;

import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;

/**
 * @author Valley
 * Created by Valley on 2023/8/29 11:21
 */
public class TextUtil {

    public static List<String> FILTER_WORD = Stream.of(
            "!", "\"", "\\", "'", "&", "#", "-", "_", "+",
            "[", "]", "【", "】", "(", ")", "（", "）", "{", "}",
            ";", ":", "/", "?", "？", "$", "~", "=", "*", "^",
            "`", "￥", ",", ".", "。", "、", "|", ">", "<"
            ).collect(Collectors.toList());

    static {
        List<String> list = FileIOUtil.readLines("filter.txt");
        FILTER_WORD.addAll(list);
    }

    /**
     * 功能：实现结巴分词效果，未过滤
     */
    public static List<String> analysis(String originText) {
        JiebaSegmenter segment = new JiebaSegmenter();
        return segment.sentenceProcess(originText);
    }

    /**
     * 功能：实现结巴分词效果，默认过滤
     */
    public static List<String> analysisFilter(String originText) {
        return analysisFilter(originText, FILTER_WORD);
    }

    /**
     * 功能：实现分词，指定过滤文件，一行一个违禁词
     */
    public static List<String> analysisFilter(String originText, String filterFile) {
        List<String> list = FileIOUtil.readLines(filterFile);
        return analysisFilter(originText, list);
    }

    /**
     * 功能：实现分词，指定过滤文件
     */
    public static List<String> analysisFilter(String originText, List<String> strings) {
        return analysis(originText).stream().map(String::trim)
                .filter(s->!strings.contains(s))
                .collect(Collectors.toList());
    }




}
